#!/usr/bin/perl
# 
# ## Note: this is numbered wrong in the first printing - it should be
# ## Exercise 10.9 not Exercise 10.10
# 
# Extract all the words (excluding words like "the" or other unnecessary words) from 
# the annotation of a library of GenBank records. For each word found, add the off-
# set of the GenBank record in the library to a DBM file that has keys equal to the 
# words, and values that are strings with offsets separated by spaces. In other words, 
# one key can have a space-separated list of offsets for a value. Then you can quickly 
# find all records containing a word like "fibroblast" with a simple lookup, followed 
# by extracting the offsets and seeking into the library with those offsets. How big is 
# your DBM file compared to the GenBank library? What might be involved in con-
# structing a search engine for the annotations in all of GenBank? For human DNA 
# only?
# 

# Answer to Exercise 10.9
#
# This exercise points out the importance of dealing with natural
# languages (like English, as opposed to Perl) in biology.  A great deal of what a biologist
# does is to read literature, such as published papers, or in this case the annotation that
# comes in a genetic database.  The ability to extract important information from literature
# is a speciality in bioinformatics.  This exercise gives you a taste of the problem.
#
# How to proceed?  We're going to do a minimal solution to the problem; to properly parse
# a natural language is a chief concern of the computer science specialty Natural
# Language Processing, and there are a lot of techniques one could use.  But we'll be
# doing something fairly simple, that will have lots of room for improvement.  Perhaps you'd
# like to take our solution and improve it!!
#
# The indexing of GenBank libraries has been shown in Chapter 10.  The approach to extracting
# all the words follows the approach in Christiansen and Torkington's Perl Cookbook, which
# I highly recommend as a source of solutions for the up-and-coming Perl programmer and for the
# seasoned professional alike.

use strict;
use warnings;
use BeginPerlBioinfo;

# The GenBank library
my $genbank_library = 'library.gb';

# The filehandle for the GenBank library
my $gbfh;

unless(open($gbfh,"$genbank_library")) {
	print "Cannot open GenBank library $genbank_library\n";
	exit;
}

# A hash to store the byte offsets of records, keyed by a word appearing in the record
my %words = ();

# A clearly inadequate list of words to ignore (can you do better?)
my %ignore = (
	'and' => 1,
	'or' => 1,
	'organism' => 1,
	'id' => 1,
	'if' => 1,
	'species' => 1,
);

# Get the GenBank records and their byte offsets
for( my $byteoffset = tell($gbfh); my $record = get_next_record($gbfh) ; $byteoffset = tell($gbfh)) {

	# Extract annotation
	my ($annotation, undef) = get_annotation_and_dna($record);
	
	# Only need to handle each word once per GenBank record
	my %seen = ();

	# Extract words, saving the byte offset for found words
	#  !! What is the definition of a word here?  Can you think of a better one for GenBank annotations? !!
	while($annotation =~ /(\w[\w'-]*)/g) {

		# Store everything in lowercase
		my $word = lc $1;

		# Skip unwanted words, or words already found in this record
		defined $ignore{$word} and next;
		defined $seen{$word} and next;

		# Mark this new word as seen
		$seen{$word} = 1;

		# Add the byte offset of this record to the value for this word in the hash
		if(defined $words{$word}) {
			$words{$word} .= " $byteoffset";
		}else{
			$words{$word} = $byteoffset;
		}
	}
}

	
#
# Interact with the user, asking for words and showing the GenBank records containing them
#

while( my $query = lc getuserinput("Search in the GenBank library $genbank_library for what word?: ") ) {

	# Ask again if the requested word doesn't appear in the library
	unless(defined $words{$query}) {
		print "The word \"$query\" is not in the library $genbank_library\n";
		next;
	}

	# If the user wants to see any of the records for the requested word
	if( getuserinput("Display the records containing that word? (y or n): ") =~ /^\s*y/i) {

		# Make an array of the byte offsets of the GenBank records containing the query word
		my @offsets = split " ", $words{$query};

		# Display each GenBank record beginning at a saved byte offset in the library
		do {
			my $offset = shift @offsets; 

			# Point the filehandle at the offset
			seek($gbfh, $offset, 0);

			# Print the record at that offset
			print get_next_record($gbfh);

		# If there is another record, and if the user wants to see it, loop again
		}while( @offsets and (getuserinput("Would you like to see the next record? (y or n): ") =~ /^\s*y/i)); 
	}
}

close($gbfh);

exit;

################################################################################
# Subroutines
################################################################################
#
# getuserinput
#  - prompt the user for a one-line response
#    return null if response is an empty line, q, or quit
#    Otherwise, return answer (with leading and trailing whitespace removed)
#
sub getuserinput {
	my($prompt) = @_;

	print $prompt;
	my $ans = <STDIN>;
	chomp $ans;
	if($ans =~ /^\s*$/ or $ans =~ /^\s*q\s*$/i or $ans =~ /^\s*quit\s*$/i) {
		return '';
	}else{
		$ans =~ s/^\s*//;
		$ans =~ s/\s*$//;
		return $ans;
	}
}
